package domain.service;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.*;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.regex.*;

import javax.jdo.JDOHelper;
import javax.jdo.PersistenceManager;
import javax.jdo.Query;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import com.google.appengine.api.datastore.Text;

import domain.data.QnAPersistenceManager;
import domain.model.Domain;
import domain.model.InOutLink;
import domain.model.Keyword;

public class SelfSiteUpdate {
	public SelfSiteUpdate(String name_domain)
	{
		try {
			PersistenceManager psm = QnAPersistenceManager.get().getPersistenceManager();
			URL dataURL = new URL("http://"+name_domain);			
			String charset   = "UTF-8";			
			name_domain = name_domain.replaceAll("www.", "");			
		    HttpURLConnection connection = (HttpURLConnection) dataURL.openConnection();
		    connection.setInstanceFollowRedirects(true);
		    BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), Charset.forName(charset)));
			String content = "";
			connection.getInputStream().mark(20000);
			while(1==1)
			{
				String str = reader.readLine();
				if(str==null) break;
				content+=str;
				
			}
			
			String server = connection.getHeaderField("Server");
			String powerBy = connection.getHeaderField("X-Powered-By");
		
			Document doc = Jsoup.parse(content);
			
			int size = doc.html().length();
			
			Elements link = doc.select("a");
			
			final int index = doc.select("meta").toString().toLowerCase( ).indexOf( "charset=" );
			if(index != -1)
				charset = doc.select("meta").toString().substring( index+8 ).replaceAll("[^\\\'\\\"0-9a-zA-Z\\-\\_]+[\\w\\W]+", "").replaceAll("[\\\'\\\"]+", "");

			connection.getInputStream().reset();
			reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), Charset.forName(charset)));

			content = "";			
			while(1==1)
			{
				String str = reader.readLine();
				if(str==null) break;
				content+=str;				
			}
			doc = Jsoup.parse(content);			
			String title = "";
			String desc=null;
			String key = null;
			int mdesc = 0;
			int mkey = 0;
			try
			{				
				try {
					title = doc.select("title").first().text();
				} catch (Exception e) {
					// TODO: handle exception
				}

				try {
					key = doc.select("meta[name=keywords]").first().attr("content");
					mkey = 1;
				} catch (Exception e) {
					try {
						key = doc.select("meta[http-equiv=keywords]").first().attr("content");
					} catch (Exception e2) {
						// TODO: handle exception
					}					
				}				
				try {
					desc = doc.select("meta[name=description]").first().attr("content");
					mdesc = 1;
				} catch (Exception e) {
					try {
						desc = doc.select("meta[http-equiv=description]").first().attr("content");
					} catch (Exception e2) {
						try {
							if(doc.select("body").first().text().length() > 260)
							{
								desc = doc.select("body").first().text().substring(0,250).replaceAll("[ ]+[\\w\\W]{0,10}+$", "");
							} else {
								desc = doc.select("body").first().text();
							}
						} catch (Exception e3) {
							// TODO: handle exception
						}
					}
				}
				if(desc!=null&&desc.length()>260)
				{
					desc = desc.substring(0,250);
				}
			}
			catch(Exception ex)
			{
				ex.printStackTrace();
				
			}
			
			int begin = content.indexOf("_gaq.push(['_setAccount', '");
			String ananyticAccount = "";
			if(begin>0)
			{
				begin+="_gaq.push(['_setAccount', '".length();
				int end =  content.indexOf("'",begin);
				ananyticAccount = content.substring(begin, end);
				begin=ananyticAccount.lastIndexOf("-")-1;
				
				ananyticAccount = ananyticAccount.substring(0, begin);			
			}
			else
			{				
				begin = content.indexOf("_gaq.push([\"_setAccount\", \"");
				if(begin>0)
				{
					begin+="_gaq.push([\"_setAccount\", \"".length();
					int end =  content.indexOf("\"",begin);
					ananyticAccount = content.substring(begin, end);
				} else {
					begin = content.indexOf("_uacct");
					if(begin > 0)
					{
						begin+="_uacct".length();
						ananyticAccount = content.substring(begin,begin+30).replaceAll("^[ =\\'\\\"]+", "").replaceAll("[^a-zA-Z0-9\\-\\_]+[\\w\\W]+", "");
						begin=ananyticAccount.lastIndexOf("-");						
						ananyticAccount = ananyticAccount.substring(0, begin);
					} else {
						Pattern p = Pattern.compile("UA-[0-9]+");
				        Matcher m = p.matcher(content);
				        while (m.find()) {
				        	ananyticAccount = m.group();
				        }
					}
				}
			}
			begin = content.indexOf("google_ad_client = \"");
			String adsAccount = "";
			if(begin>0)
			{
				begin+="google_ad_client = \"', '".length();
				int end =  content.indexOf("\"",begin);
				adsAccount = content.substring(begin, end);
			}
			
			PersistenceManager psms = QnAPersistenceManager.get().getPersistenceManager();
			ArrayList<String> l_keywords = new ArrayList<String> ();
			@SuppressWarnings("unchecked")
			List<Domain> domainList=  (List<Domain>)psm.newQuery(Domain.class,"name == '"+name_domain+"'").execute();
			if(domainList.size()>0)
			{
				Domain domain = domainList.get(0);
				
				domain.setDomain(Utils.returnDomain(name_domain));
				
				domain.setName(name_domain);
				domain.setWebserver(server);
				domain.setProgramingLanguage(powerBy);
				domain.setTitle(title);
				if(desc != null)
					domain.setDescription(new Text(desc));
				
				if(key!=null)
				{
					String[] keywords = key.split(",");
					for(int i=0;i<keywords.length;i++)
					{
						if(!l_keywords.contains(keywords[i].trim()))
							l_keywords.add(keywords[i].trim());
						
						Query query1 = psms.newQuery(Keyword.class);
						query1.setFilter("name=='"+keywords[i].trim()+"' && domain=='"+name_domain+"'");
						@SuppressWarnings("unchecked")
						List<Keyword> check_key = (List<Keyword>) query1.execute();
						if(check_key.size() <= 0)
						{
							Keyword insert_key = new Keyword();
							insert_key.setName(keywords[i].trim());
							insert_key.setDomain(name_domain);
							psms.makePersistent(insert_key);
						}
					}
				}
				domain.setKeywords(l_keywords);
				domain.setGoogleAnalytic(ananyticAccount);
				domain.setGoogleAdsense(adsAccount);
				
				try
				{
					// email				
					final String RE_MAIL = "([\\w\\-]([\\.\\w])+[\\w]+@([\\w\\-]+\\.)+[A-Za-z]{2,4})";
				    Pattern p = Pattern.compile(RE_MAIL);
				    Matcher m = p.matcher(content.replaceAll("\\<.*?\\>", " "));
	
				    while(m.find()) {
				    	if(m.group(1).endsWith(name_domain))
				    	{
	//			    		domain.addEmail(m.group(1).toLowerCase());				    		
				    	}				    	
				    } 
				}
				catch (Exception ex)
				{
					 ex.printStackTrace();
					 
				}
				
				// JS
				ArrayList<String> l_js = new ArrayList<String> ();
				if(content.indexOf("jquery")>0)
					l_js.add("jquery");
				if(content.indexOf("prototype.js")>0)
					l_js.add("prototype");
				if(content.indexOf("extjs")>0)
					l_js.add("extjs");
				if(content.indexOf("yui")>0)
					l_js.add("yui");
				if(content.indexOf("dojo")>0)
					l_js.add("dojo");
				if(content.indexOf("gwt")>0)
					l_js.add("gwt");
				if(content.indexOf("easeljs")>0)
					l_js.add("easeljs");
				
				domain.setJsLib(l_js);
				//phan tich the
				ArrayList<String> l_h1 = new ArrayList<String> ();
				Elements element = doc.select("h1");
				String save;
				for(int i=0;i<element.size();i++)
				{
					if(element.get(i).text().length() > 500)
						save = element.get(i).text().substring(0,500);
					else 
						save = element.get(i).text();
					l_h1.add(save);
				}
				domain.setH1(l_h1);
				ArrayList<String> l_h2 = new ArrayList<String> ();
				element = doc.select("h2");
				for(int i=0;i<element.size();i++)
				{
					if(element.get(i).text().length() > 500)
						save = element.get(i).text().substring(0,500);
					else 
						save = element.get(i).text();
					l_h2.add(save);
				}
				domain.setH2(l_h2);
				ArrayList<String> l_h3 = new ArrayList<String> ();
				element = doc.select("h3");
				for(int i=0;i<element.size();i++)
				{
					if(element.get(i).text().length() > 500)
						save = element.get(i).text().substring(0,500);
					else 
						save = element.get(i).text();
					l_h3.add(save);
				}
				domain.setH3(l_h3);
				ArrayList<String> l_h4 = new ArrayList<String> ();
				element = doc.select("h4");
				for(int i=0;i<element.size();i++)
				{
					if(element.get(i).text().length() > 500)
						save = element.get(i).text().substring(0,500);
					else 
						save = element.get(i).text();
					l_h4.add(save);
				}
				domain.setH4(l_h4);
				ArrayList<String> l_h5 = new ArrayList<String> ();
				element = doc.select("h5");
				for(int i=0;i<element.size();i++)
				{
					if(element.get(i).text().length() > 500)
						save = element.get(i).text().substring(0,500);
					else 
						save = element.get(i).text();
					l_h5.add(save);
				}
				domain.setH5(l_h5);
				ArrayList<String> l_img = new ArrayList<String> ();
				element = doc.select("img");
				for(int i=0;i<element.size();i++)
				{
					if(Pattern.matches("http[\\w\\W]+",element.get(i).attr("src")))
					{
						if(element.get(i).attr("src").length() > 500)
							save = element.get(i).attr("src").substring(0,500);
						else 
							save = element.get(i).attr("src");
						l_img.add(save);
					} else {
						save = "http://"+name_domain+"/"+element.get(i).attr("src").replaceAll("^[\\/]+", "");
						if(save.length() > 500)
							save = save.substring(0,500);
						l_img.add(save);
					}
				}
				domain.setImg(l_img);
				ArrayList<String> l_flash = new ArrayList<String> ();
				element = doc.select("embed");
				for(int i=0;i<element.size();i++)
				{
					if(Pattern.matches("http[\\w\\W]+",element.get(i).attr("src")))
					{
						if(element.get(i).attr("src").length() > 500)
							save = element.get(i).attr("src").substring(0,500);
						else 
							save = element.get(i).attr("src");
						l_flash.add(save);
					} else {
						save = "http://"+name_domain+"/"+element.get(i).attr("src").replaceAll("^[\\/]+", "");
						if(save.length() > 500)
							save = save.substring(0,500);
						l_flash.add(save);
					}
				}
				domain.setFlash(l_flash);
				domain.setFrame(doc.select("frame").size());
				domain.setCharset(charset);
				
				dataURL = new URL("http://api.mywot.com/0.4/public_query2?url=http://"+domain.getName());

				connection = (HttpURLConnection) dataURL.openConnection();
				
				connection.setReadTimeout(500000);
				connection.setConnectTimeout(1000000);
				
				reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), Charset.forName("UTF-8")));
				content = "";
				while(1==1)
				{
					String str = reader.readLine();
					if(str==null) break;
					content+=str;
					
				}
				
				doc = Jsoup.parse(content);
				Elements checks = doc.select("application");
				String str = title;
				int sex = 0;
				str = str.replaceAll("[^\\W]+sex[^\\W]+", " ");
				str = str.replaceAll("sex[^\\W]+", " ");
				str = str.replaceAll("[^\\W]+sex", " ");
				str = str.replaceAll("[^\\W]+porn[^\\W]+", " ");
				str = str.replaceAll("porn[^\\W]+", " ");
				str = str.replaceAll("[^\\W]+porn", " ");
				str = str.toLowerCase();
				if(str.indexOf("sex")>0 || str.indexOf("porn")>0)
				{
					sex = 1;
				}
				str = desc;
				str = str.replaceAll("[^\\W]+sex[^\\W]+", " ");
				str = str.replaceAll("sex[^\\W]+", " ");
				str = str.replaceAll("[^\\W]+sex", " ");
				str = str.replaceAll("[^\\W]+porn[^\\W]+", " ");
				str = str.replaceAll("porn[^\\W]+", " ");
				str = str.replaceAll("[^\\W]+porn", " ");
				str = str.toLowerCase();
				if(str.indexOf("sex")>0 || str.indexOf("porn")>0)
				{
					sex = 1;
				}
	
				if(checks.size() > 0 && checks.size() == 4)
				{
					int check = Integer.parseInt(doc.select("application").last().attr("r").trim());
					domain.setField8(check);
					if(check < 50)
					{
						domain.setSex(1);
					}
				}
				if(sex == 1)
				{
					domain.setSex(1);
				}
				domain.setMetaDesc(mdesc);
				domain.setMetaKey(mkey);
				domain.setSize(size);
				psm=JDOHelper.getPersistenceManager(domain);
				psm.currentTransaction().begin();
				psm.makePersistent(domain);
				psm.currentTransaction().commit();
			}
			
			ArrayList<String> listLink = new ArrayList<String>();
			String check = null;
			PersistenceManager psm1 = QnAPersistenceManager.get().getPersistenceManager();
			for(int i=0;i<link.size();i++)
			{
				check = link.get(i).attr("rel"); 
				if(Pattern.matches("^http[\\w\\W]+", link.get(i).attr("href")))
				{
					if( check != null && !check.equals("nofollow") || check == null)
					{
						Pattern p = Pattern.compile("^http[^\\/]+[\\/]{2}[^\\/]+");
	
				        Matcher m = p.matcher(link.get(i).attr("href"));
				        while (m.find()) {
				        	String dom = m.group().replaceAll("(http|https)+://(www.)*", "").replaceAll("[\\?]+[\\w\\W]+", "").replaceAll("[\\/]+[\\w\\W]+", "");
				        	if(!listLink.contains(dom) && !dom.equals(name_domain))
				        	{
				        		listLink.add(dom);
				        		Query query = psm1.newQuery(InOutLink.class);
				        		query.setFilter("inLink=='"+name_domain+"' && outLink=='"+dom+"'");
				        		@SuppressWarnings("unchecked")
								List<InOutLink> checks = (List<InOutLink>) query.execute();
				        		if(checks.size() <=0)
				        		{
				        			InOutLink outlinks = new InOutLink();
				        			outlinks.setInLink(name_domain);
				        			outlinks.setOutLink(dom);
				        			psm1.makePersistent(outlinks);
				        		}
				        	}
				        }
					}
				}
			}

		}
		catch (Exception e) {
			
			e.printStackTrace();
		} 
		
	}
//	public static void main( String args[])
//	{
//		new SelfSiteUpdate("mintworks.com");
//	}
}
